from scrapy import Request
from scrapy.spiders import Spider
from lianjia_home.items import LianjiaHomeItem

class LianjiaSpider(Spider):
    name = "lianjia"
    current_page = 1
    total_page = 0

    def start_requests(self):
        start_url = "https://su.lianjia.com/ershoufang/"
        yield Request(start_url)

    def parse(self, response):
        list_selector = response.xpath("//div[@class='info clear']")
        for one_selector in list_selector:
            name = one_selector.xpath("div[@class='flood']/div/a[@data-el='region']/text()").extract_first()
            other_info = one_selector.xpath("div[@class='address']/div/text()").extract_first()
            info_list = other_info.split("|")
            type = info_list[0].strip(" ")
            area = info_list[1].strip(" ")
            direction = info_list[2].strip(" ")
            fitment = info_list[3].strip(" ")
            price = one_selector.xpath("div[@class='priceInfo']//span/text()").extract()
            total_price = price[0]
            unit_price = price[1]

            item = LianjiaHomeItem()
            item["name"] = name.strip()
            item["type"] = type
            item["area"] = area
            item["direction"] = direction
            item["fitment"] = fitment
            item["total_price"] = total_price
            item["unit_price"] = unit_price

            # 生成详细页的请求
            detail_url = one_selector.xpath("div[@class='title']/a/@href").extract_first()
            yield Request(detail_url, meta = {"item": item}, callback = self.detail_parse)

        # 获取下一页
        if self.total_page == 0:
            page_data = response.xpath("//div[@class='page-box fr']/div/@page-data").extract_first()
            self.total_page = eval(page_data)["totalPage"]
        self.current_page += 1
        if self.current_page <= self.total_page:
            next_url = "https://su.lianjia.com/ershoufang/pg%d" % self.current_page
            yield Request(next_url)

    def detail_parse(self, response):
        elevator = response.xpath("//div[@class='content']/ul/li[last()]/text()").extract_first()

        item = response.meta["item"]
        item["elevator"] = elevator

        yield item